#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/11/13 8:54
# @Author  : Yunhao.Cao
# @File    : demo_spider.py
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy_demo.items import DemoItem, DetailItem

__author__ = 'Yunhao.Cao'

__ALL__ = []


class DemoSpider(Spider):
    name = "demo"
    start_urls = [
        "https://creditcard.bankcomm.com/content/pccc/discount/activitylist/more.html",
    ]

    host = "https://creditcard.bankcomm.com"

    def parse(self, response):
        filename = response.url.split("/")[-1]
        with open(filename, 'wb') as f:
            f.write(response.body)
        title = response.selector.xpath("/html/head/title/text()").extract()[0]
        name = "demo"

        item = DemoItem()
        item["title"] = title
        item["name"] = name

        yield item

        url_list = response.xpath("//div[@class='actlist']/ul/li/a/@href").extract()

        for url in url_list[:2]:
            yield Request(url=self.host + url, callback=self.parse_detail)

    def parse_detail(self, response):
        filename = response.url.split("/")[-1]
        with open(filename, 'wb') as f:
            f.write(response.body)

        title = response.selector.xpath("/html/head/title/text()").extract()[0]
        content = response.selector.xpath("//div[@class='thsh_mods']").xpath("string(.)").extract()[0]

        item = DetailItem()
        item["title"] = title
        item["content"] = content

        yield item


if __name__ == '__main__':
    pass
